package com.wzgl.core.webmagic.processor;

import java.io.Serializable;
import java.util.Date;
import java.util.List;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import com.jfinal.kit.StrKit;
import com.jfinal.plugin.redis.Redis;
import com.wzgl.core.CacheConsts;
import com.wzgl.core.model.Tgrid;
import com.wzgl.core.utils.DateUtil;
import com.wzgl.core.vo.MeiWenVo;
import com.wzgl.core.webmagic.MagicUtils;
import com.wzgl.core.webmagic.pipeline.TgridsPipeLine;

/**
 * @author code4crafter@gmail.com <br>
 * @since 0.6.0
 */
public class MeiwenPageProcessorTest extends MeiWenVo implements PageProcessor,Serializable,Job {
	/**
	 * 
	 */
	private static final long serialVersionUID = 1L;
	/***
	 * addHeader 网站默认是zip模式 如果网站没有使用zip 则需要增加Header
	 */
    private Site site = Site.me().setRetryTimes(1).setSleepTime(1000).addHeader("Accept-Encoding", "/");
    //图片前缀
    String pic_ = "http://meiwen.59xihuan.cn";
    static Spider s ;
    int size = 1;
    Tgrid t = null;
    StringBuffer sb = new StringBuffer();
    @Override
    public void process(Page page) {
    	 if(page.getUrl().regex("http://meiwen.59xihuan.cn/meiwen").match()){
    		 System.out.println("获得meiwen_1_URL....size:"+size);
    		 size++;
    	 }else{
    		 List<String> page_left= page.getHtml().xpath("//div[@class='main']/div[@class='mLeft']/div[@class='post']").all();
    		 int num=1;
        	 for(String li:page_left){
        		 System.out.println("获取1_URL详情页....num:"+num);
        		 num++;
        	 }
    	 }
    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
    	 /**
    	  * 单独使用 redis缓存机制
    	  */
	     
        s = Spider.create(new MeiwenPageProcessorTest()).addUrl(getFurl()).thread(getFthread());    
    	s.addPipeline(new TgridsPipeLine());
    	s.run();
    }

	@Override
	public void execute(JobExecutionContext context) throws JobExecutionException {
		// TODO Auto-generated method stub
		s = Spider.create(new MeiwenPageProcessorTest()).addUrl("http://meiwen.59xihuan.cn").thread(1);    
//    	s.addPipeline(new TgridsPipeLine());s
    	s.run();
	}
}